work_dir=/data/git/gfdmp_for_mljr/
. ${work_dir}/function/logging.sh

function f_mysql_backup_init_var()
{
	#moshan:初始化备份的超时及锁的变量
	if [ "${ftwrl_time}x" != "x" ]
	then
		backup_init_var[0]="--ftwrl-wait-timeout=${ftwrl_time}"
		backup_init_var[1]="--ftwrl-wait-threshold=10"
		backup_init_var[2]="--ftwrl-wait-query-type=all"
	else
		backup_init_var[0]="--kill-long-queries-timeout=20"
		backup_init_var[1]="--kill-long-query-type=select"
	fi
}

function f_write_backup_status_to_mysql()
{
	#moshan:将备份状态写进mysql相应的表，作为监控记录
	#moshan:0表示成功，1表示失败，2表示备份成功，但是转储到nas失败
	r_user="user_name"
	r_pass="password"
	r_host="127.0.0.1"
	r_port="3306"
	r_db="test_db"
	r_date="$(date +%F)"
	r_size="$(du -sm ${current_full_backup_dir}/${mysql_backup_file}|awk '{if($1>=1024) {print $1/1024"G"}else {print $1"M"}}')"
	r_time="${backup_ok_time}"
	r_status="${1}"
	r_comm="mysql -u${r_user} -p${r_pass} -h${r_host} -P${r_port} ${r_db}"
	if [ "${project_id}x" != "x" ]
	then
		[ "${backup_mode}x" == "fullx" ] && backup_mode_tmp="FULL" || backup_mode_tmp="INCREMENTAL"
		insert_comm="echo 'insert into backup_info_tmp_for_ms(name,type,backup_time,size,is_unzip,time_cost,status,project_port,project_name) select \"${mysql_backup_file}\",\"${backup_mode_tmp}\",\"${r_date}\",\"${r_size}\",\"True\",\"${r_time}\",${r_status},\"${mysql_port}\",(select name from backup_project where id = ${project_id});'|${r_comm} 2>/dev/null"
		if [ "${debug}x" == "1x" ]
		then
			f_logging "BACKUP" "$(echo ${insert_comm}|sed "s/-p.* -h/-p****** -h/g")"|tee -a ${log_file}
		fi
		#echo "insert into backup_info_tmp_for_ms(name,type,backup_time,size,is_unzip,time_cost,status,project_port,project_name) select \"${mysql_backup_file}\",\"${backup_mode_tmp}\",\"${r_date}\",\"${r_size}\",\"True\",\"${r_time}\",${r_status},\"${mysql_port}\",(select name from backup_project where id = ${project_id});"|${r_comm} 2>/dev/null
		eval ${insert_comm}
		if [ $? -eq 0 ]
		then
			f_logging "INFO" "The backup status [${r_status}] has been written to mysql."|tee -a ${log_file}
		else
			f_logging "WARN" "Backup status [${r_status}] failed to write to mysql."|tee -a ${log_file}
		fi
	fi
}

function f_mysql_backup()
{
	if [ -f "${HOME}/mysql_backup_${mysql_port}" ]
	then
		f_logging "WARN" "There are already other processes backing up the instance[MySQL:${mysql_port}], please do not repeat the backup." "0"|tee -a ${log_file}
		f_logging "WARN" "Manually delete files [${HOME}/mysql_backup_${mysql_port}] if there is no backup process."|tee -a ${log_file}
		exit
	else
		touch ${HOME}/mysql_backup_${mysql_port}
	fi
	backup_mode="$(awk -F= '{print $2}' <<< "${1}")" #moshan:接收备份模式
	f_mysql_backup_init_var
	f_test_var #moshan:检查mysql相关的变量
	f_logging "INFO" "MySQL data ${backup_mode} backup starting, and will allocate ${use_mem} memory..."|tee -a ${log_file}
	#moshan:判断用户是否正确配置了mysql_data_dir_filesystem变量
	#moshan:检查磁盘空间, 检查数据目录所在的分区的剩余空间是否大于当前需要恢复的mysql的数据目录的大小
	mysql_data_dir_space="$(du -sm ${mysql_data_dir} 2> /dev/null|awk '{print $1/1024}')"
	mysql_backup_space="$(awk '{print $1/3}' <<< "${mysql_data_dir_space}")"
	free_space="$(df|grep ${mysql_data_dir_filesystem} 2> /dev/null|awk '{print $4/1024/1024}')"
	if [ "${free_space}x" == "x" ]
	then
		[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
		f_logging "ERROR" "FileSystem \"[${mysql_data_dir_filesystem}]\" does not exist" "2" "0"|tee -a ${log_file}
		return
	else
		f_logging "INFO" "FileSystem \"[${mysql_data_dir_filesystem}]\" disk space:${free_space}G"|tee -a ${log_file}	
		f_logging "INFO" "MySQL data dir space:${mysql_data_dir_space}G"|tee -a ${log_file}	
		f_logging "INFO" "This backup may require one third of the remaining space of the \"${mysql_data_dir_filesystem}\" partition:${mysql_backup_space}G"|tee -a ${log_file}	
		free_space="$(awk -F. '{print $1}' <<< ${free_space})"
		mysql_backup_space="$(awk -F. '{print $1}' <<< ${mysql_backup_space})"
		if [ "$(echo "$((${free_space}-${mysql_backup_space}))"|grep -c "-")x" == "1x" ]
		then
			#moshan:判断磁盘空间
			[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
			f_logging "ERROR" "FileSystem \"[${mysql_data_dir_filesystem}]\" not enough disk space" "2" "0"|tee -a ${log_file}
			return
		fi
	fi
	unset mysql_data_dir_space mysql_backup_space free_space free_space

	#moshan:判断是否是slave, 判断sql线程是否是running状态
	sql_thread_is_run="$(${mysql_base_dir}/bin/mysql -u${mysql_backup_user} -p${mysql_backup_passwd} -S ${mysql_sock} -e "show slave status\G" 2>/dev/null|grep "Slave_SQL_Running:"|awk -F'Slave_SQL_Running: ' '{print $2}')"
	if [ "${sql_thread_is_run}x" == "Yesx" ]
	then
		#moshan:sql线程是running状态, 则backup_slave_info参数进行配置
		backup_slave_info="--slave-info --safe-slave-backup --safe-slave-backup-timeout=120"
	else
		#moshan:sql线程不是running状态, 则backup_slave_info参数不进行配置
		backup_slave_info=""
	fi
		backup_slave_info="--slave-info --safe-slave-backup --safe-slave-backup-timeout=120"

	current_full_backup_dir="${current_backup_dir}" #moshan:定义当前全备的目录路径
	[ ! -d "${backup_dir}" ] && mkdir -p ${backup_dir} #moshan:如果不存在则创建该目录
    from_lsn_log_file="${current_backup_dir}/from_lsn.log"
	current_full_backup_dir=${current_backup_dir}
	mkdir -p ${current_backup_dir}
	if [ "${backup_encrypt}x" == "1x" ]
	then
		if [ "${encrypt_passwd}x" != "x" ]
		then
			encrypt_passwd_tmp=$(md5sum <<< "${encrypt_passwd}"|awk '{print $1}')
		else
			encrypt_passwd_tmp=$(md5sum <<< "${current_backup_dir}/${mysql_full_backup_file}"|awk '{print $1}')
		fi
		encrypt_tmp="--encrypt=AES256 --encrypt-key=${encrypt_passwd_tmp} --encrypt-threads=${threads}"
	else
		encrypt_tmp=""
	fi
	command_tmp="innobackupex --defaults-file=${mysql_conf} --password='${mysql_backup_passwd}' --socket=${mysql_sock} --user=${mysql_backup_user} --parallel=${threads} --compress --compress-threads=${threads} --compress-chunk-size=2M ${encrypt_tmp} --throttle=400 ${backup_slave_info} --no-timestamp --stream=xbstream --use-memory=${use_mem} --tmpdir=${backup_tmp_dir1} --extra-lsndir='${current_backup_dir}' ${backup_init_var[*]} --no-version-check ${current_backup_dir} > ${current_backup_dir}/${mysql_full_backup_file}"
	if [ "${backup_mode}x" == "fullx" ]
	then
		#moshan:如果是全备操作，则具体操作如下
		if [ "${debug}x" == "1x" ]
		then
			f_logging "BACKUP" "$(echo ${command_tmp}|sed "s/--password='.*' --socket/--password='******' --socket/g"|sed "s/--encrypt-key=.* --throttle=/--encrypt-key='******' --throttle=/g")"|tee -a ${log_file}
		fi
		eval ${command_tmp}
        if [ $? -eq 0 ]
		then
			#moshan:备份命令返回正确的状态值，则标志备份执行成功，并将backup_state变量赋值为1
			backup_state=1
			#moshan:全备文件的名称，用于日志打印.
			mysql_backup_file="${mysql_full_backup_file}" 
			#moshan:判断当前备份目录的备份数量是否大于定义的最多保留的份数
			backup_count_tmp=$(ls -rt ${backup_dir}|grep -vc tmp) 
			if [ ${backup_count_tmp} -gt ${backup_count} ]
			then
				#moshan:若当前备份目录的备份数量大于定义的最多保留的份数，则删除较旧的备份
				tmp=$((${backup_count_tmp}-${backup_count}))
				cd ${backup_dir}
				ls -rt ${backup_dir}|grep -v tmp|head -${tmp}|xargs -i rm -rf {}
			fi
		else
			#moshan:备份失败的话，删除本次备份生成的相应文件
			[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
			[ -f "${current_backup_dir}/${mysql_full_backup_file}" ] && rm -f ${current_backup_dir}/${mysql_full_backup_file}
			[ -d "${current_backup_dir}" ] && rmdir ${current_backup_dir}
		fi
	else
		if [ ! -f "${current_backup_dir}/xtrabackup_checkpoints" ]
		then
			mkdir -p ${current_backup_dir}
			#moshan:判断全备目录是否存在（用户指定的目录），如果不存在则使用备份目录最新的一份全备
			last_backup_dir="${backup_dir}/$(ls -lrt ${backup_dir} 2>/dev/null|grep -Ev "tmp|^-"|tail -2|head -1|awk '{print $NF}')"
			if [ ! -d "${last_backup_dir}" ]
			then
				#moshan:如果备份目录为空, 则创建备份目录
				f_logging "WARN" "The specified backup directory \"${current_backup_dir}\" does not exist and will use the directory of the latest full backup file." "2"|tee -a ${log_file}
				f_logging "WARN" "Current instance backup is empty. Now creating a backup directory [\"${current_backup_dir}\"]"|tee -a ${log_file}
				current_full_backup_dir="${current_backup_dir}"
			else
				#moshan: 否则使用最新一次备份文件所在的目录
				f_logging "WARN" "The specified backup directory \"${current_backup_dir}\" does not exist and will use the directory \"${last_backup_dir}\" of the latest full backup file." "2"|tee -a ${log_file}
				[ -d "${current_backup_dir}" ] && rmdir ${current_backup_dir} 2>/dev/null 
				current_full_backup_dir="${last_backup_dir}"
				current_backup_dir="${current_full_backup_dir}"
			fi
		else
			#moshan:判断全备目录是否存在（用户指定的目录），如果存在则使用.
			current_full_backup_dir="${current_backup_dir}"
		fi
		if [ ! -f "${current_full_backup_dir}/xtrabackup_checkpoints" ]
		then
			#moshan:判断全备目录是否是正确的全备文件,判断条件是判断xtrabackup_checkpoints文件是否存在
			#moshan:如果不存在，则判断force_full_backup变量的配置，如果不为1则直接退出系统
			#moshan:如果force_full_backup变量的配置为1，则做一次全备
			if [ "${force_full_backup}x" != "1x" ]
			then
				f_logging "ERROR" "MySQL data ${backup_mode} backup error, the reason is Full file is error, force_full_backup!=1 and EXIT!" "2" "0"|tee -a ${log_file}
				[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
				return
			else
				[ -d "${current_backup_dir}" ] && rm -rf ${current_backup_dir}/*
				backup_mode="full"
				f_logging "WARN" "MySQL data ${backup_mode} backup error, the reason is Full file is error, but force_full_backup=1. Now,the program needs to empty the \"${current_backup_dir}\" directory and starting full backup!" "2" "0"|tee -a ${log_file}
				if [ "${debug}x" == "1x" ]
				then
					f_logging "BACKUP" "$(echo ${command_tmp}|sed "s/--password='.*' --socket/--password='******' --socket/g")"|tee -a ${log_file}
				fi
				eval ${command_tmp}
				if [ $? -eq 0 ]
				then
					#moshan:备份命令返回正确的状态值，则标志备份执行成功，并将backup_state变量赋值为1
					backup_state=1
					#moshan:全备文件的名称，用于日志打印.
					mysql_backup_file="${mysql_full_backup_file}" 
					#moshan:判断当前备份目录的备份数量是否大于定义的最多保留的份数
					backup_count_tmp=$(ls -rt ${backup_dir}|grep -vc tmp) 
					if [ ${backup_count_tmp} -gt ${backup_count} ]
					then
						#moshan:若当前备份目录的备份数量大于定义的最多保留的份数，则删除较旧的备份
						tmp=$((${backup_count_tmp}-${backup_count}))
						cd ${backup_dir}
						ls -rt ${backup_dir}|grep -v tmp|head -${tmp}|xargs -i rm -rf {}
					fi
				else
					#moshan:备份失败的话，删除本次备份生成的相应文件
					[ -f "${current_backup_dir}/${mysql_full_backup_file}" ] && rm -f ${current_backup_dir}/${mysql_full_backup_file}
					[ -d "${current_backup_dir}" ] && rmdir ${current_backup_dir}
					[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
				fi
			fi
		else
            from_lsn_log_file="${current_full_backup_dir}/from_lsn.log"
			#moshan:如果是增备操作，且全备状态正常，则保存上一次全备的多间
			#moshan:checkpoints_file_create_time="2018-09-19 14:42:00"
			checkpoints_file_create_time_tmp="$(stat ${current_full_backup_dir}/xtrabackup_checkpoints 2> /dev/null|grep "Modify"|awk '{print $2" "$3}'|sed 's#\..*##g')"
			if [ "${checkpoints_file_create_time_tmp}x" == "x" ]
			then
				checkpoints_file_create_time="$(stat ${current_full_backup_dir}/xtrabackup_checkpoints 2> /dev/null|grep "最近更改"|awk '{print $1" "$2}'|awk -F '最近更改：' '{print $2}'|sed 's#\..*##g')"
			else
				checkpoints_file_create_time="${checkpoints_file_create_time_tmp}"
			fi
			#moshan:full_file_create_time=1537339320, 换成秒数-5
			full_file_create_time="$(($(date +%s -d "${checkpoints_file_create_time}")-5))"
			#moshan:full_file_create_time="2018-09-19 14:41:55", 再换成时间格式
			full_file_create_time="$(date +"%F %T" -d @${full_file_create_time})"
			checkpoints_file_create_time="$(($(date +%s -d "${checkpoints_file_create_time}")+5))"
			checkpoints_file_create_time="$(date +"%F %T" -d @${checkpoints_file_create_time})"
			#moshan:最终结果是full_file_create_time比xtrabackup_checkpoints文件修改时间少5秒
			#moshan:最终结果是checkpoints_file_create_time比xtrabackup_checkpoints文件修改时间多5秒
			#moshan:相当于预留十秒的空闲时间，用于后面的查找，即查找这个时间段的全备文件（当一个目录下存在多份全备文件的时候有用）
			#moshan:因为备份文件与xtrabackup_checkpoints文件几乎是同一时刻创建的。
			#moshan:下面查找就是新于full_file_create_time时间的文件，且旧于checkpoints_file_create_time时间的备份文件
			#moshan:找到于xtrabackup_checkpoints文件最近的一份全备文件，并记录备份的时间作为组名
			increm_group="$(find ${current_full_backup_dir} -type f \( -newermt "${full_file_create_time}" -a -not -newermt "${checkpoints_file_create_time}" \)|grep $(cut -c $(($(echo ${#mysql_port})+1))-17 <<< "${mysql_full_backup_file}")|awk -F'_' '{print $NF}')"
			if [ "${increm_group}x" == "x" ]
			then
				#moshan:上面找increm_group的时候，如果当前备份目录存在增量文件，则不适用，需要用下面的
				increm_group="$(find ${current_full_backup_dir} -type f \( -newermt "${full_file_create_time}" -a -not -newermt "${checkpoints_file_create_time}" \)|grep $(cut -c $(($(echo ${#mysql_port})+1))-28 <<< "${mysql_increm_backup_file}")|awk -F'_' '{print $6}'|grep -v "log$"|tail -1)"
			fi
			#moshan:定义增量文件的名称
			mysql_increm_backup_file_tmp="${mysql_increm_backup_file}_${increm_group}"
			#moshan:找到最后一份增量的编号, 即增备的命名是累加
			last_increm_backup_file=$(cat ${from_lsn_log_file} 2>/dev/null|tail -1)
			last_increm_backup_file=$(awk -F_ '{print $7}' <<< "${last_increm_backup_file}")
			if [ "${last_increm_backup_file}x" != "x" ]
			then
				count=$((${last_increm_backup_file}+1))
			else
				count=1
			fi
			command_tmp="innobackupex --defaults-file=${mysql_conf} --user=${mysql_backup_user} --password='${mysql_backup_passwd}' --socket=${mysql_sock} --incremental --incremental-basedir=${current_full_backup_dir} --parallel=${threads} --compress --compress-threads=${threads} --compress-chunk-size=2M ${encrypt_tmp} ${backup_slave_info} --throttle=400 --no-timestamp --stream=xbstream --use-memory=${use_mem} --tmpdir=${backup_tmp_dir2} ${backup_init_var[*]} --extra-lsndir='${current_full_backup_dir}' --no-version-check ${current_full_backup_dir} > ${current_full_backup_dir}/${mysql_increm_backup_file_tmp}_${count}"
			if [ "${debug}x" == "1x" ]
			then
				f_logging "BACKUP" "$(echo ${command_tmp}|sed "s/--password='.*' --socket/--password='******' --socket/g")"|tee -a ${log_file}
			fi
			eval ${command_tmp}
			if [ $? -eq 0 ]
			then
				#moshan:备份命令返回正确的状态值，则标志备份执行成功，并将backup_state变量赋值为1
				backup_state=1
				#moshan:全备文件的名称，用于日志打印.
				mysql_backup_file="${mysql_increm_backup_file_tmp}_${count}"
			else
			    #moshan:备份失败的话，删除本次备份生成的相应文件
				if [ -f "${current_full_backup_dir}/${mysql_increm_backup_file_tmp}_${count}" ]
				then
					rm -f ${current_full_backup_dir}/${mysql_increm_backup_file_tmp}_${count}
					[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
				fi
			fi
		fi
	fi
	if [ "${backup_state}x" == "1x" ]
	then
        from_lsn="$(grep "from_lsn = " ${current_full_backup_dir}/xtrabackup_checkpoints 2>/dev/null|tr -d " "|awk -F= '{print $2}')"
        echo "${from_lsn}:${current_full_backup_dir}/${mysql_backup_file}" >> ${from_lsn_log_file}
		backup_state=0
		#moshan:备份成功，并备份mysql的配置文件
		backup_ok_time="$(date "+%H:%M:%S")"
		[ -f "${mysql_conf}" ] && cp ${mysql_conf}  ${current_backup_dir}/my.cnf.${mysql_port}
		[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
		f_logging "INFO" "MySQL data ${backup_mode} backup successed! The file is ${current_full_backup_dir}/${mysql_backup_file}"|tee -a ${log_file}
		if [ -d "${remote_backup_dir}" ] 
		then
			f_logging "INFO" "Copying backup file [${current_full_backup_dir}/${mysql_backup_file}] to ${remote_backup_dir}..."|tee -a ${log_file}
			cp ${current_full_backup_dir}/${mysql_backup_file} ${remote_backup_dir} 
			if [ $? -eq 0 ]
			then
				f_logging "INFO" "Copying backup file [${current_full_backup_dir}/${mysql_backup_file}] to ${remote_backup_dir} successfully..."|tee -a ${log_file}
				f_write_backup_status_to_mysql 0
			else
				[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
				f_logging "ERROR" "Failed to copy backup file [${current_full_backup_dir}/${mysql_backup_file}] to ${remote_backup_dir}..." "2" "0"|tee -a ${log_file}
				f_write_backup_status_to_mysql 2
			fi
		fi
		if [ "${send_mark}x" == "1x" ]
		then
			#moshan:如果用户指定了需要传输到远程的备份服务器，则向下进行相应的操作
			for ((ip_list=0;ip_list<=${#backup_host[@]};ip_list++))
			do
				if [ "${backup_host}x" == "x" ]
				then
					#moshan:配置文件如果没有配上备份机的ip列表，则转储失败，并终止传输
					f_logging "WARN" "Sending backup file to remote server, but remote server ip list is empy, and EXIT..."|tee -a ${log_file}
					break
				fi
				ping -c 1 ${backup_host[${ip_list}]} > /dev/null 2>&1
				ping_mark=$?
				if [ "${send_backup_file_force}x" == "1x" ]
				then
					#moshan:判断备份机是否可ping通，若ping不通
					#moshan:配置文件send_backup_file_force配置为1，则打印提示信息，并继续传输
					f_logging "WARN" "The host[${backup_host[${ip_list}]}] unreachable, but send_backup_file_force=1. Now Try sending backup file to ${backup_host[${ip_list}]}"|tee -a ${log_file}
					ping_mark=0
				fi
				if [ "${ping_mark}x" == "0x" ]
				then
					f_logging "INFO" "Sending backup file to ${backup_host[${ip_list}]}..."|tee -a ${log_file}
				    scp -P${remote_host_port} -r ${current_full_backup_dir} ${remote_host_user}@${backup_host[${ip_list}]}:${backup_dir}
					if [ $? -eq 0 ]
					then
						#moshan:传输scp状态判断
						f_logging "INFO" "Sending backup file to ${backup_host[${ip_list}]} successfully..."|tee -a ${log_file}
					else
						f_logging "ERROR" "Failed to send backup file to ${backup_host[${ip_list}]} ..." "2" "0"|tee -a ${log_file}
						[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
					fi
				else
					#moshan:判断备份机是否可ping通，若ping不通
					#moshan:配置文件send_backup_file_force配置为非1，则打印错误信息，并终止传输
					f_logging "ERROR" "The host[${backup_host[${ip_list}]}] unreachable, and failed to send backup file to ${backup_host[${ip_list}]} ..." "2" "0"|tee -a ${log_file}
					[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
				fi
			done
		fi
	else
		#moshan:备份失败，并打印错误信息
		[ -f "${HOME}/mysql_backup_${mysql_port}" ] && rm -f ${HOME}/mysql_backup_${mysql_port}
		f_logging "ERROR" "MySQL data ${backup_mode} backup error!" "2" "1"|tee -a ${log_file}
		f_write_backup_status_to_mysql 1
	fi
}
